package com.cosmo.integration.service;

import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.message.BasicNameValuePair;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import org.springframework.xml.transform.StringResult;
import org.springframework.xml.xpath.XPathExpression;
import org.springframework.xml.xpath.XPathExpressionFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import com.cosmo.integration.web.HttpResponseResult;

public class AtomServiceImpl extends BaseService implements AtomService {

    private final Log logger = LogFactory.getLog(getClass());

    private final XPathExpression selectBaseAttFromServiceElement;
    private final XPathExpression selectBaseAttFromFeedElement;
    private final XPathExpression selectBaseAttFromEntryElement;

    private final XPathExpression selectHomeWorkspace;
    private final XPathExpression selectCollectionIds;

    private final DocumentBuilderFactory documentBuilderFactory;
    private final TransformerFactory transformerFactory;

	DateTimeFormatter dateFormatter = ISODateTimeFormat.dateTimeNoMillis();
	Charset utf8 = Charset.forName("UTF-8");

	public AtomServiceImpl(String serviceBaseScheme, String serviceBaseHost,
    							int serviceBasePort, String serviceBasePath,
                                HttpClient httpClient, Map<String, String> namespaces,
                                String serviceFrontUrl) {

    	super(serviceBaseScheme, serviceBaseHost, serviceBasePort, serviceBasePath, httpClient, namespaces, serviceFrontUrl);
        this.documentBuilderFactory = DocumentBuilderFactory.newInstance();
        this.documentBuilderFactory.setNamespaceAware(true);

        this.transformerFactory = TransformerFactory.newInstance();

        // match the root (/) for 'service' (app is the namespace) and the attribute 'xml:base'
        this.selectBaseAttFromServiceElement = XPathExpressionFactory.createXPathExpression("/app:service/@xml:base", namespaces);
        // match the root element 'feed' (atom is the namespace) and the attribute 'xml:base'
        this.selectBaseAttFromFeedElement = XPathExpressionFactory.createXPathExpression("/atom:feed/@xml:base", namespaces);
        // same for the <entry node which occurs on a saveEvent return
        this.selectBaseAttFromEntryElement = XPathExpressionFactory.createXPathExpression("/atom:entry/@xml:base", namespaces);
        // get the 'home' workspace as opposed to 'account' workspace
        // '//' matches from the current node all 'atom:title' elements with a text node of home, then gets its parent (..)
        this.selectHomeWorkspace = XPathExpressionFactory.createXPathExpression("//atom:title[text()='home']/..", namespaces);
        // then get the collection href's from the home workspace
        // matches current node app:collection (app namespace) with href attribute
        this.selectCollectionIds = XPathExpressionFactory.createXPathExpression("./app:collection/@href", namespaces);
    }


	// the CosmoFeedServiceSpec - described http://chandlerproject.org/Projects/CosmoFeedServiceSpec
    // which we have fleshed out into an atom cosmo-interface

    // entry point feed service - /atom/user/{username}
	// /atom/user/{username} - user service document - lists the uris of users collections, subscriptions and preferences
	// workspace 'home' returned: collections published in home (no 'accept' element, since not updateble itself)
	// workspace 'account' returned: atom:title subscribed and atom:title preferences
    //    * 200 : Success
	//    * 500 : Unknown server error generating the service document
    // this gets the collections (in home workspace) and subscriptions, preferences (in account workspace)
    public HttpResponseResult getWorkspaces(String username) {
    	String path = getServiceBasePath() + "/user/" + username;
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
    	return translateBaseUrl(sendReceive(get, true));
    }
    // returns subscriptions - collections and tickets
    // http://chandlerproject.org/Projects/CosmoFeedSubscriptions
	//    * 200: Success
	//    * 500: Unknown server error
    public HttpResponseResult getSubscriptions(String username) {
    	String path = getServiceBasePath() + "/user/" + username + "/subscriptions";
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
        return translateFeedUrl(sendReceive(get, true));
    }
    // http://chandlerproject.org/Projects/CosmoFeedPreferences
    //    * 200: Success
    //	  * 500: Unknown server error
    public HttpResponseResult getPreferences(String username) {
    	String path = getServiceBasePath() + "/user/" + username + "/preferences";
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
        return translateFeedUrl(sendReceive(get, true));
    }

    // collection feed service - /atom/collection/{uuid}[/{projection}[/{format}]][?query string]
    // projection is amount of data
    //		- basic, inline human readable - intended for read only in a hCalendar format defined through xmdp
    //		- full, everything
    //		- dashboard, like full
    //		- dashboard-now/later/done, like full, but limited to 'done, now, later', ignores time ranges, but supports timezones
    //		- details, extra info about collections (not their items)

    // lists members of a collection (as a collection feed)
    // 'get'
	// 		- content collection is an atom collection, notes are atom entries as EIM data serialized to xml or json
	//		- ordered, recent first
	//		- EIM - external information model - a central model all others can translate to - http://chandlerproject.org/Projects/SharingProject
	// 		- the collections can be time-range queried and filtered according to projections
	//		- collection attributes are represented in xhtml in a format defined by an xmdp - http://gmpg.org/xmdp/
	//		- collection items are the items
	//		- reply includes a etag and last-modified for caching
	//		- reply is application/atom+xml in representation requested
	//    * 200: Success
	//    * 400: Invalid query parameters, or unsupported projection or data format
	//    * 500: Unknown server error
    public HttpResponseResult getDetailsForCollection(String collectionId) {
    	String path = getServiceBasePath() + "/collection/" + collectionId + "/details";
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
        return translateFeedUrl(sendReceive(get, true));
    }
	// content collection is an atom collection, notes are atom entries as EIM data serialized to xml or json
    // the errors are as above - this is just a representation request
    public HttpResponseResult getDashboardForCollection(String collectionId) {
    	String path = getServiceBasePath() + "/collection/" + collectionId + "/dashboard/eim-json";
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
        return translateFeedUrl(sendReceive(get, true));
    }
    // the RFC 3339 date-time, and allows tz - the olson timezone identifier
    // the errors are as above - this is just a representation request 
    public HttpResponseResult getEventsInPeriod(String collectionId, DateTime start, DateTime end) {
    	String path = getServiceBasePath() + "/collection/" + collectionId + "/full/eim-json";
        String startStr = convertToCosmoDate(start);
        String endStr = convertToCosmoDate(end);
        List <NameValuePair> nvps = new ArrayList <NameValuePair>();
        nvps.add(new BasicNameValuePair("start", startStr));
        nvps.add(new BasicNameValuePair("end", endStr));
        HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, nvps, null);
        return translateFeedUrl(sendReceive(get, true));
        /* http client 3.1
    	public String getEventsInPeriod(String collectionId, DateTime start, DateTime end) {
    	String url = serviceBaseUrl + "/collection/" + collectionId + "/full/eim-json";
        GetMethod get = new GetMethod(url);
        String startStr = convertToCosmoDate(start);
        String endStr = convertToCosmoDate(end);
        NameValuePair[] params = {new NameValuePair("start", startStr), new NameValuePair("start", endStr)};
        get.setQueryString(params);
        setStandardHeaders(get);
        return translateFeedUrl(sendReceive(get, passBackErrors));
        */
    }
    // collection feed service (even though under the user uri)
    // create a collection in the home collection with the uuid
    // 'post'ed xhtml representation of collection using users service uri (uuid is not required)
    // response is blank, but the headers include an etag and last-modified for caching, and location is the new collection
    // response codes:
	//    * 201: The collection was created successfully
	//    * 400: The request content could not be parsed or contained an invalid representation
	//    * 409: An item with the specified UUID already exists in the server
	//    * 411: The request Content-Length header was not provided or was not greater than 0
	//    * 415: The request Content-Type header was not provided or was not application/xhtml+xml
	//    * 423: The home collection was locked and could not be updated
	//    * 500: Unknown server error
    public HttpResponseResult createCollection(String username, String collectionName, String uuid) {
        String createTemplate = "<?xml version='1.0' encoding='UTF-8'?><div class=\"collection\"><span class=\"name\">TEMPLATE_COLLECTION_NAME</span\n" +
                				"><span class=\"uuid\">TEMPLATE_UUID</span></div>";
        String body = createTemplate.replace("TEMPLATE_COLLECTION_NAME", collectionName).replace("TEMPLATE_UUID", uuid);
    	String path = getServiceBasePath() + "/user/" + username;
    	HttpPost post = createPost(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, body, null);
    	// "requests must be application/xhtml_xml" - http://chandlerproject.org/Projects/CosmoFeedCollections
        setContentTypeHeader(post, "application/xhtml+xml");
    	return sendReceive(post, true);
        /* retained since multiple refactoring
        try {
            PostMethod postCollection = new PostMethod(url);
            postCollection.setRequestEntity(new StringRequestEntity(createTemplate, "application/xhtml+xml; charset=UTF-8", "UTF-8"));
            postCollection.setRequestHeader("Content-Type", "application/xhtml+xml");
            postCollection.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
            // the return can be null, http/1.1 204 No Content - but still deleted
            sendReceive(postCollection);
        } catch (UnsupportedEncodingException unsE) {
            throw new RuntimeException(unsE);
        }
        */
    }
	//    * 204: Update applied successfully
	//    * 411: The request Content-Length header was not provided or was not greater than 0
	//    * 415: The request Content-Type header was not provided or was not application/xhtml+xml
	//    * 423: The collection was locked and could not be updated
	//    * 500: Unknown server error
    public HttpResponseResult updateCollection(String newCollectionName, String collectionId) {
        String createTemplate = "<?xml version='1.0' encoding='UTF-8'?><div class=\"collection\"><span class=\"name\">TEMPLATE_COLLECTION_NAME</span\n" +
                				"></div>";
        String body = createTemplate.replace("TEMPLATE_COLLECTION_NAME", newCollectionName);
    	String path = getServiceBasePath() + "/collection/" + collectionId;
    	HttpPut put = createPut(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, body, null);
    	// "requests must be application/xhtml_xml" - http://chandlerproject.org/Projects/CosmoFeedCollections
        setContentTypeHeader(put, "application/xhtml+xml");
        return sendReceive(put, true);
        /* retained since multiple refactoring
        try {
            PostMethod postCollection = new PostMethod(url);
            postCollection.setRequestEntity(new StringRequestEntity(createTemplate, "application/xhtml+xml; charset=UTF-8", "UTF-8"));
            postCollection.setRequestHeader("Content-Type", "application/xhtml+xml");
            postCollection.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
            // the return can be null, http/1.1 204 No Content - but still deleted
            sendReceive(postCollection);
        } catch (UnsupportedEncodingException unsE) {
            throw new RuntimeException(unsE);
        }
        */
    }
	//    * 204: Success
	//    * 423: The collection was locked and could not be deleted
	//    * 500: Unknown server error 
    public HttpResponseResult deleteCollection(String collectionId) {
    	String path = getServiceBasePath() + "/collection/" + collectionId + "/full/eim-json";
    	HttpDelete delete = createDelete(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
        return sendReceive(delete, true);
    }


    // item feed service - /atom/item/{uuid}[/{projection}[/{format}]][?{query string}]
    //		- projection is the authorisation method? - 'basic' 
    // can be manipulated using app (Atom Publishing Protocol - AtomPub or APP)
    // content is provided in eim-json (json), or eimml (xml)  - http://chandlerproject.org/Projects/CosmoFeedCollectionsAndItems#Data%20Formats
    // reply is atom document - application/atom+xml, including an etag and last-modified
	//    * 200: Success
	//    * 400: Unsupported projection or data format
	//    * 500: Unknown server error 
    public HttpResponseResult getEvent(String itemId, boolean fromCosmo) {
    	String path = getServiceBasePath() + "/item/" + itemId;
    	HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, null, null);
    	boolean passBackErrors = false;
    	if (fromCosmo)
    		passBackErrors = true;
    	return translateEntryUrl(sendReceive(get, passBackErrors));
    }
    // create - uses the collection uri
    // does not store meta data
    // response includes an atom document of the item of application/atom+xml with headers
    // response also includes a content-location to tell client that representation is complete
    //    * 201: The item was successfully created
	//    * 400: The request content could not be parsed, did not represent an Atom entry or contained an invalid representation
	//    * 409: An item with the specified UUID already exists in the server, or the uploaded item contained an iCalendar UID that is in use by an item already in the collection
	//    * 411: The request Content-Length header was not provided or was not greater than 0
	//    * 415: The request Content-Type header was not provided or was not application/atom+xml
	//    * 423: The collection was locked and could not be updated
	//    * 500: Unknown server error 
    public HttpResponseResult createEvent(String collectionId, String jsonEvent, boolean fromCosmo) {
    	String path = getServiceBasePath() + "/collection/" + collectionId + "/full/eim-json";
    	HttpPost post = createPost(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, jsonEvent, null);
        setJsonHeaders(post);
    	boolean passBackErrors = fromCosmo;
    	HttpResponseResult result = translateEntryUrl(sendReceive(post, passBackErrors));
    	return result;
        /* retained since multiple refactoring
        String url = serviceBaseUrl + "/collection/" + collectionId + "/full/eim-json";
    	try {
    		PostMethod postEvent = new PostMethod(url);
    		postEvent.setRequestEntity(new StringRequestEntity(jsonEvent, "application/atom+xml; charset=UTF-8", "UTF-8"));
    		postEvent.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
    		return translateEntryUrl(sendReceive(postEvent));
    	} catch (UnsupportedEncodingException unsE) {
    		throw new RuntimeException(unsE);
    	}
    	*/
	}
    // add event to an existing collection
	//    * 204: The item was successfully added to the collection
	//    * 400: The request content did not contain a data pair with name uuid and non-empty value, or an item with that uuid could not be found, or the item with that uuid is not a note
	//    * 411: The request Content-Length header was not provided or was not greater than 0
	//    * 423: The collection was locked and could not be updated
	//    * 500: Unknown server error
    public HttpResponseResult addEvent(String collectionId, String itemId, boolean fromCosmo) {
    	String path = getServiceBasePath() + "/collection/" + collectionId;
        List <NameValuePair> nvps = new ArrayList <NameValuePair>();
        nvps.add(new BasicNameValuePair("uuid", itemId));
    	HttpPost post = createPost(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, nvps, null);
    	// must be application/x-www-form-urlencoded - hence the post method above
    	boolean passBackErrors = false;
    	if (fromCosmo)
    		passBackErrors = true;
    	return sendReceive(post, passBackErrors);
	}
	//    * 200: The item was successfully modified
	//    * 400: The request content could not be parsed, did not represent an Atom entry or contained an invalid representation
	//    * 409: The item contained an iCalendar UID that is in use by an item already in the collection
	//    * 411: The request Content-Length header was not provided or was not greater than 0
	//    * 415: The request Content-Type header was not provided or was not application/atom+xml
	//    * 423: The collection was locked and could not be updated
	//    * 500: Unknown server error 
    public HttpResponseResult updateEvent(String itemId, String jsonEvent, boolean fromCosmo) {
    	String path = getServiceBasePath() + "/item/" + itemId + "/full/eim-json";
    	HttpPut put = createPut(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, jsonEvent, null);
        setJsonHeaders(put);
    	boolean passBackErrors = false;
    	if (fromCosmo)
    		passBackErrors = true;
    	return translateEntryUrl(sendReceive(put, passBackErrors));
        /* retained since multiple refactoring
    	String url = serviceBaseUrl + "/item/" + collectionId + "/full/eim-json";
    	try {
    		PutMethod event = new PutMethod(url);
    		event.setRequestEntity(new StringRequestEntity(jsonEvent, "application/atom+xml; charset=UTF-8", "UTF-8"));
    		event.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
    		return translateEntryUrl(sendReceive(event));
    	} catch (UnsupportedEncodingException unsE) {
    		throw new RuntimeException(unsE);
    	}
    	*/
	}
    // removes from every collection - unless a uuid is provided
	//    * 204: Success
	//    * 409: The item for the provided UUID was not found or is not a collection
	//    * 423: The collection was locked and could not be updated
	//    * 500: Unknown server error 
    public HttpResponseResult deleteEvent(String collectionId, String uuid, boolean fromCosmo) {
    	String path = getServiceBasePath() + "/item/" + collectionId + "/full/eim-json";
    	List <NameValuePair> nvps = null;
    	if (StringUtils.isNotBlank(uuid)) {
    		nvps = new ArrayList <NameValuePair>();
        	nvps.add(new BasicNameValuePair("uuid", uuid));
    	}
    	HttpDelete delete = createDelete(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, nvps, null);
    	boolean passBackErrors = false;
    	if (fromCosmo)
    		passBackErrors = true;
        return sendReceive(delete, passBackErrors);
        /* retained since multiple refactoring
    	String url = serviceBaseUrl + "/item/" + collectionId + "/full/eim-json";
        DeleteMethod del = new DeleteMethod(url);
        NameValuePair[] params = {new NameValuePair("uuid", uuid)};
        del.setQueryString(params);
        setStandardHeaders(del);
        sendReceive(del);
        */
    }
	// /atom/expanded/{uuid}[/{projection}[/{format}]][?{query string}].
	// gets the expanded version of a recurring item - within a specific time, or just the item itself
	//    * 200: Success
	//    * 400: Invalid query parameters, or unsupported projection or data format
	//    * 500: Unknown server error 
    public HttpResponseResult getExpandedEventInPeriod(String itemId, DateTime start, DateTime end) {
    	String path = getServiceBasePath() + "/expanded/" + itemId;
    	List <NameValuePair> nvps = null;
    	if ((start != null) && (end != null)) {
    		String startStr = convertToCosmoDate(start);
    		String endStr = convertToCosmoDate(end);
    		nvps = new ArrayList <NameValuePair>();
    		nvps.add(new BasicNameValuePair("start", startStr));
    		nvps.add(new BasicNameValuePair("end", endStr));
    	}
        HttpGet get = createGet(getServiceBaseScheme(), getServiceBaseHost(), getServiceBasePort(), path, nvps, null);
        return translateFeedUrl(sendReceive(get, true));
    }

    protected HttpResponseResult translateBaseUrl(HttpResponseResult result) {
    	result.setResponseBody(translateBaseUrl(result.getResponseBody(), selectBaseAttFromServiceElement));
    	return result;
    }
    protected HttpResponseResult translateFeedUrl(HttpResponseResult result) {
    	result.setResponseBody(translateBaseUrl(result.getResponseBody(), selectBaseAttFromFeedElement));
    	return result;
    }
    protected HttpResponseResult translateEntryUrl(HttpResponseResult result) {
    	result.setResponseBody(translateBaseUrl(result.getResponseBody(), selectBaseAttFromEntryElement));
    	return result;
    }
    protected String translateBaseUrl(String responseAsString, XPathExpression matchElement) {
    	try {
            DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder();
            // strings are internally encoded utf-16
            // TODO are the other objects encoded?
            org.w3c.dom.Document responseDoc = builder.parse(new InputSource(new StringReader(responseAsString)));
            Node baseAtt = matchElement.evaluateAsNode(responseDoc);
            baseAtt.setNodeValue(getServiceFrontUrl());

            StringResult backAsString = new StringResult();
            transformerFactory.newTransformer().transform(new DOMSource(responseDoc), backAsString);
            logger.info("Passing back translated response ");
            logger.info(backAsString);

            return backAsString.toString();
        } catch (Exception e) {
            throw new RuntimeException("Conversion of response failed, invalid response?", e);
        }
    }

    // for TESTING
    public List<String> getHomeCollectionIds(String username) {
    	HttpResponseResult result = getWorkspaces(username);
    	Element homeWorkspace = getHomeWorkSpaceElement(result.getResponseBody());
        List<Node> ids = selectCollectionIds.evaluateAsNodeList(homeWorkspace);

        List<String> cutIdsList = new ArrayList<String>();
        for (Node id : ids) {
            String idStr = id.getTextContent();
            String cutId = idStr.substring(idStr.indexOf('/') + 1);
            cutIdsList.add(cutId);
        }

        return cutIdsList;
    }
    // supplementary for testing
    protected Element getHomeWorkSpaceElement(String responseAsString) {
        Document doc = getDocumentForString(responseAsString);
        return (Element) selectHomeWorkspace.evaluateAsNode(doc);
    }
    // supplementary for testing
    protected Document getDocumentForString(String responseAsString) {
        try {
            DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder();
            return builder.parse(new InputSource(new StringReader(responseAsString)));
        } catch (Exception e) {
            throw new RuntimeException("Conversion of response failed, invalid response?", e);
        }
    }

/* rewritten jonas extended code
    // internal methods which submit a request to the cosmo server
    protected String getResponseForUrl(String url) {
    	return getResponseForUrl("get", url);
    }
    protected String getResponseForUrl(String methodStr, String url) {
    	return getResponseForUrl(methodStr, url, null);
    }
    protected String getResponseForSubmission(String url, String command) {
		HttpPost method = new HttpPost(url);
		//byte[] bytes = EncodingUtil.getBytes(command, "UTF-8");
		//RequestEntity entity = new ByteArrayRequestEntity(bytes); 
		//((PostMethod) method).setRequestEntity(entity);
		try {
            method.setRequestEntity(new StringRequestEntity(command, "application/xhtml+xml; charset=UTF-8", "UTF-8"));
            method.setRequestHeader("Content-Type", "application/xhtml+xml");
            method.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
		} catch (UnsupportedEncodingException e) {
			logger.error("failed setting post data: " + e);
		}
		return sendReceive(method);
	}
    protected String getResponseForUrl(String methodStr, String url, String command) {
    	GetMethod method = null;
   		method = new GetMethod(url);
   		setStandardHeaders(method);
		return sendReceive(method);
    }
	*/

/* redundant in httpclient 4
	protected String sendReceiveResponse(HttpMethodBase method) throws IOException {

		// get the stream of bytes (to avoid a system default character encoding)
		// getBodyAsString uses the incoming charset, but may not be specified, and could be too long a response for a string
		String charset = method.getRequestCharSet();
		// decode - we assume utf-8 (even though spec says iso-8859-1 - we'd rather keep everything utf-8)
		if (StringUtils.isEmpty(charset))
			charset = "UTF-8";
		BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream(), charset));
		// TODO not sure why we read a line at a time, because reading x bytes is surely better?
		String inputLine = "";
        StringBuilder responseBody = new StringBuilder();
        while ((inputLine = in.readLine()) != null) {
        	// because we read a line, we need to put a line back in
        	responseBody.append(inputLine);//.append(System.getProperty("line.separator"));
        }
        in.close();
        String response = responseBody.toString();
        return response;
	}
	*/

/* http client 3.1
    public void createCollection(String collectionName, String uuid) {
        String createTemplate = "<?xml version='1.0' encoding='UTF-8'?><div class=\"collection\"><span class=\"name\">TEMPLATE_COLLECTION_NAME</span\n" +
                				"><span class=\"uuid\">TEMPLATE_UUID</span></div>";
        createTemplate = createTemplate.replace("TEMPLATE_COLLECTION_NAME", collectionName).replace("TEMPLATE_UUID", uuid);
        String url = serviceBaseUrl + "/user/" + username;

        try {
            PostMethod postCollection = new PostMethod(url);
            postCollection.setRequestEntity(new StringRequestEntity(createTemplate, "application/xhtml+xml; charset=UTF-8", "UTF-8"));
            postCollection.setRequestHeader("Content-Type", "application/xhtml+xml");
            // FIXME - had to change this since disturbed comments, was star/star
            postCollection.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
            // the return can be null, http/1.1 204 No Content - but still deleted
            sendReceive(postCollection);
        } catch (UnsupportedEncodingException unsE) {
            throw new RuntimeException(unsE);
        }
    }
    public void deleteCollection(String collectionId) {
        String url = serviceBaseUrl + "/collection/" + collectionId + "/full/eim-json";
        DeleteMethod del = new DeleteMethod(url);
        setStandardHeaders(del);
        sendReceive(del);
    }
    public String createEvent(String collectionId, String jsonEvent) {
    	String url = serviceBaseUrl + "/collection/" + collectionId + "/full/eim-json";
    	try {
    		PostMethod postEvent = new PostMethod(url);
    		postEvent.setRequestEntity(new StringRequestEntity(jsonEvent, "application/atom+xml; charset=UTF-8", "UTF-8"));
    		// FIXME as above
    		postEvent.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
    		return translateEntryUrl(sendReceive(postEvent));
    	} catch (UnsupportedEncodingException unsE) {
    		throw new RuntimeException(unsE);
    	}
	}
    public String updateEvent(String collectionId, String jsonEvent) {
    	String url = serviceBaseUrl + "/item/" + collectionId + "/full/eim-json";
    	try {
    		PutMethod event = new PutMethod(url);
    		event.setRequestEntity(new StringRequestEntity(jsonEvent, "application/atom+xml; charset=UTF-8", "UTF-8"));
    		// FIXME as above
    		event.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
    		return translateEntryUrl(sendReceive(event));
    	} catch (UnsupportedEncodingException unsE) {
    		throw new RuntimeException(unsE);
    	}
	}
    public void deleteEvent(String collectionId, String uuid) {
    	String url = serviceBaseUrl + "/item/" + collectionId + "/full/eim-json";
        DeleteMethod del = new DeleteMethod(url);
        NameValuePair[] params = {new NameValuePair("uuid", uuid)};
        del.setQueryString(params);
        setStandardHeaders(del);
        sendReceive(del);
    }
    // internal methods which submit a request to the cosmo server
    protected String getResponseForUrl(String url) {
    	return getResponseForUrl("get", url);
    }
    protected String getResponseForUrl(String methodStr, String url) {
    	return getResponseForUrl(methodStr, url, null);
    }
    protected String getResponseForSubmission(String url, String command) {
		PostMethod method = new PostMethod(url);
		//byte[] bytes = EncodingUtil.getBytes(command, "UTF-8");
		//RequestEntity entity = new ByteArrayRequestEntity(bytes); 
		//((PostMethod) method).setRequestEntity(entity);
		try {
            method.setRequestEntity(new StringRequestEntity(command, "application/xhtml+xml; charset=UTF-8", "UTF-8"));
            method.setRequestHeader("Content-Type", "application/xhtml+xml");
            // FIXME as above
            method.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
		} catch (UnsupportedEncodingException e) {
			logger.error("failed setting post data: " + e);
		}
		return sendReceive(method);
	}
    protected String getResponseForUrl(String methodStr, String url, String command) {
    	GetMethod method = null;
   		method = new GetMethod(url);
   		setStandardHeaders(method);
		return sendReceive(method);
    }

    protected void setStandardHeaders(HttpMethod method) {
        // we prefer to receive back text/html and application/xhtml+xml, then application/xml or anything else
    	// FIXME as above
    	method.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,star/star;q=0.8");
        // cosmo uses the header text/xml
        // but to avoid the problems with this we specify the charset in the http header
    	// and we need to specify it explicitly, because our character filter doesn't touch this internal request
    	// http://annevankesteren.nl/2005/03/text-xml
        method.setRequestHeader("Content-Type", "text/xml; charset=UTF-8");
        // indicate we are a javascript request
        method.setRequestHeader("X-Requested-With", "XMLHttpRequest");
    }

    protected URI createURI(HttpServletRequest request) {

		// with the query string, most urlencoders actually handle only the query part (or the application/x-www-form-urlencoded)
		// however, whilst we assumed the encoding in the path was ok, our httpclient 3.1 sent a + in the query string part - which is wrong - it needs to be %2b as cosmo expects
		// so we upgraded, and changed the path encoding in the process

    	// see http://www.lunatech-research.com/archives/2009/02/03/what-every-web-developer-must-know-about-url-encoding
    	// we need to be able to correctly encode a uri - decoding is not a problem since the servlet can interpret it
		// because we have path information which is not correctly encoded in java
    	// the solution in constructing a uri is to encode each part
    	// we trust the new httpclient 4 to do this correctly
    	// we also trust that httpclient 4 obeys the query string law of encoding + into %2b which cosmo expects
    	// in fact, the only thing we need here is that the collectionId is correctly encoded in the path part of the request
    	// so we construct and deconstruct a URI using httpclient URIUtils

    	// the returns from the methods are decoded
    	URI uri = URIUtils.createURI(request.getScheme(), request.getHost(), request.getPort(), request.getPathInfo(), query, fragment)

		// for want of a better approach for now
		// we create the uri using 
		//URI uri = new URI(url);
		//URI = URIUtils.createURI(scheme, host, port, path, query, fragment)
    	return uri;
	}

	// as copied from MobileProviderService
    // TODO we need to externalise this, since we use it here, the controller and mobileproviderservice
	protected String sendReceive(HttpMethodBase method) {
		// Provide custom retry handler is necessary
		//method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
		//method.setDoAuthentication(false);
		String response = null;
		boolean ok = true;
        try {
    		logger.info("Making " + method.getName() + " " + method.getURI());

    		// http://hc.apache.org/httpclient-3.x/charencodings.html
    		// "httpClient will do all the charset conversion for you. Just make sure the charser attribute of the Content-Type is set"
			int statusCode = httpClient.executeMethod(method);
			if ((statusCode != HttpStatus.SC_OK) && // any get
				(statusCode != HttpStatus.SC_CREATED) && // create collection
				(statusCode != HttpStatus.SC_NO_CONTENT)) // delete an event or collection
				ok = false;

			if (ok)
				response = sendReceiveResponse(method);

		} catch (HttpException e) {
			ok = false;
		} catch (IOException e) {
			ok = false;
		} finally {
			// Release the connection.
			if (method != null)
				method.releaseConnection();
		}

        logger.info("Status of response was " + method.getStatusText() + " (" + method.getStatusLine() + ")");
        if (!ok)
			logger.warn("Response failed in some way: " + method.getStatusCode());

		return response;
	}
	protected String sendReceiveResponse(HttpMethodBase method) throws IOException {

		// get the stream of bytes (to avoid a system default character encoding)
		// getBodyAsString uses the incoming charset, but may not be specified, and could be too long a response for a string
		String charset = method.getRequestCharSet();
		// decode - we assume utf-8 (even though spec says iso-8859-1 - we'd rather keep everything utf-8)
		if (StringUtils.isEmpty(charset))
			charset = "UTF-8";
		BufferedReader in = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream(), charset));
		// TODO not sure why we read a line at a time, because reading x bytes is surely better?
		String inputLine = "";
        StringBuilder responseBody = new StringBuilder();
        while ((inputLine = in.readLine()) != null) {
        	// because we read a line, we need to put a line back in
        	responseBody.append(inputLine);//.append(System.getProperty("line.separator"));
        }
        in.close();
        String response = responseBody.toString();
        return response;
	}
*/

}
